{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers.core import Flatten, Dense, Dropout\n",
    "from keras.layers.convolutional import Conv2D, MaxPooling2D, ZeroPadding2D\n",
    "from keras.optimizers import Adam\n",
    "import numpy as np\n",
    "import cv2\n",
    "import os\n",
    "from  sklearn.model_selection import train_test_split\n",
    "import glob\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('./data/train.csv')\n",
    "df_test = pd.read_csv('./data/test.csv')\n",
    "IMAGE_SIZE = 28\n",
    "IMAGE_CHANNEL = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>pixel0</th>\n",
       "      <th>pixel1</th>\n",
       "      <th>pixel2</th>\n",
       "      <th>pixel3</th>\n",
       "      <th>pixel4</th>\n",
       "      <th>pixel5</th>\n",
       "      <th>pixel6</th>\n",
       "      <th>pixel7</th>\n",
       "      <th>pixel8</th>\n",
       "      <th>...</th>\n",
       "      <th>pixel774</th>\n",
       "      <th>pixel775</th>\n",
       "      <th>pixel776</th>\n",
       "      <th>pixel777</th>\n",
       "      <th>pixel778</th>\n",
       "      <th>pixel779</th>\n",
       "      <th>pixel780</th>\n",
       "      <th>pixel781</th>\n",
       "      <th>pixel782</th>\n",
       "      <th>pixel783</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 785 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  \\\n",
       "0      1       0       0       0       0       0       0       0       0   \n",
       "1      0       0       0       0       0       0       0       0       0   \n",
       "2      1       0       0       0       0       0       0       0       0   \n",
       "3      4       0       0       0       0       0       0       0       0   \n",
       "4      0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "   pixel8    ...     pixel774  pixel775  pixel776  pixel777  pixel778  \\\n",
       "0       0    ...            0         0         0         0         0   \n",
       "1       0    ...            0         0         0         0         0   \n",
       "2       0    ...            0         0         0         0         0   \n",
       "3       0    ...            0         0         0         0         0   \n",
       "4       0    ...            0         0         0         0         0   \n",
       "\n",
       "   pixel779  pixel780  pixel781  pixel782  pixel783  \n",
       "0         0         0         0         0         0  \n",
       "1         0         0         0         0         0  \n",
       "2         0         0         0         0         0  \n",
       "3         0         0         0         0         0  \n",
       "4         0         0         0         0         0  \n",
       "\n",
       "[5 rows x 785 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = df_train['label']\n",
    "x_train = df_train.drop('label',axis=1).values.reshape(-1,IMAGE_SIZE,IMAGE_SIZE,IMAGE_CHANNEL)\n",
    "x_test = df_test.values.reshape(-1,IMAGE_SIZE,IMAGE_SIZE,IMAGE_CHANNEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def VGG_16(weights_path=None):\n",
    "    model = Sequential()\n",
    "    model.add(ZeroPadding2D((1,1),input_shape=(IMAGE_SIZE,IMAGE_SIZE,IMAGE_CHANNEL)))\n",
    "    model.add(Conv2D(32, (3, 3),activation='relu'))\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Conv2D(32, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D((2,2), strides=(2,2)))\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D((2,2), strides=(2,2)))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(256, activation='relu'))\n",
    "    model.add(Dropout(0.25))\n",
    "    model.add(Dense(256, activation='relu'))\n",
    "    model.add(Dropout(0.25))\n",
    "    model.add(Dense(10, activation='softmax'))\n",
    "    if weights_path and os.path.isfile(weights_path) is True:\n",
    "        model.load_weights(weights_path)\n",
    "        \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = VGG_16(weights_path='Digit_Recognizer_kerasvgg16_weights.h5')\n",
    "adam = Adam(lr=0.0003)\n",
    "model.compile(optimizer=adam, loss='categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"zero_padding2d_1_input:0\", shape=(?, 28, 28, 1), dtype=float32)\n",
      "Tensor(\"dense_3/Softmax:0\", shape=(?, 10), dtype=float32)\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "zero_padding2d_1 (ZeroPaddin (None, 30, 30, 1)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 28, 28, 32)        320       \n",
      "_________________________________________________________________\n",
      "zero_padding2d_2 (ZeroPaddin (None, 30, 30, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 28, 28, 32)        9248      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 14, 14, 32)        0         \n",
      "_________________________________________________________________\n",
      "zero_padding2d_3 (ZeroPaddin (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 14, 14, 64)        18496     \n",
      "_________________________________________________________________\n",
      "zero_padding2d_4 (ZeroPaddin (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 14, 14, 64)        36928     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 7, 7, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 3136)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               803072    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                2570      \n",
      "=================================================================\n",
      "Total params: 936,426\n",
      "Trainable params: 936,426\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "print(model.input)\n",
    "print(model.output)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets_series = pd.Series(y_train)\n",
    "one_hot = pd.get_dummies(targets_series, sparse = True)\n",
    "one_hot_labels = np.asarray(one_hot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 0 ... 0 0 0]\n",
      " [1 0 0 ... 0 0 0]\n",
      " [0 1 0 ... 0 0 0]\n",
      " ...\n",
      " [0 0 0 ... 1 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 1]]\n"
     ]
    }
   ],
   "source": [
    "print(one_hot_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_raw = np.array(one_hot_labels, np.uint8)\n",
    "x_train_raw = np.array(x_train, np.float32) / 255.\n",
    "x_test  = np.array(x_test, np.float32) / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(42000, 28, 28, 1)\n",
      "(42000, 10)\n",
      "(28000, 28, 28, 1)\n"
     ]
    }
   ],
   "source": [
    "print(x_train_raw.shape)\n",
    "print(y_train_raw.shape)\n",
    "print(x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_valid, Y_train, Y_valid = train_test_split(x_train_raw, y_train_raw, test_size=0.03, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40740 samples, validate on 1260 samples\n",
      "Epoch 1/5\n",
      "40740/40740 [==============================] - 139s 3ms/step - loss: 0.2398 - val_loss: 0.0804\n",
      "Epoch 2/5\n",
      "40740/40740 [==============================] - 139s 3ms/step - loss: 0.0649 - val_loss: 0.0664\n",
      "Epoch 3/5\n",
      "40740/40740 [==============================] - 138s 3ms/step - loss: 0.0463 - val_loss: 0.0531\n",
      "Epoch 4/5\n",
      "40740/40740 [==============================] - 141s 3ms/step - loss: 0.0329 - val_loss: 0.0489\n",
      "Epoch 5/5\n",
      "40740/40740 [==============================] - 139s 3ms/step - loss: 0.0265 - val_loss: 0.0391\n"
     ]
    }
   ],
   "source": [
    "model.fit(X_train, Y_train, epochs=5,batch_size=32, validation_data=(X_valid, Y_valid), verbose=1)\n",
    "model.save_weights('Digit_Recognizer_kerasvgg16_weights.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40740 samples, validate on 1260 samples\n",
      "Epoch 1/5\n",
      "40740/40740 [==============================] - 141s 3ms/step - loss: 0.0236 - val_loss: 0.0386\n",
      "Epoch 2/5\n",
      "40740/40740 [==============================] - 144s 4ms/step - loss: 0.0181 - val_loss: 0.0518\n",
      "Epoch 3/5\n",
      "40740/40740 [==============================] - 147s 4ms/step - loss: 0.0160 - val_loss: 0.0405\n",
      "Epoch 4/5\n",
      "40740/40740 [==============================] - 150s 4ms/step - loss: 0.0117 - val_loss: 0.0571\n",
      "Epoch 5/5\n",
      "40740/40740 [==============================] - 146s 4ms/step - loss: 0.0123 - val_loss: 0.0458\n"
     ]
    }
   ],
   "source": [
    "model.fit(X_train, Y_train, epochs=5,batch_size=32, validation_data=(X_valid, Y_valid), verbose=1)\n",
    "model.save_weights('Digit_Recognizer_kerasvgg16_weights.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28000/28000 [==============================] - 30s 1ms/step\n"
     ]
    }
   ],
   "source": [
    "y_test = model.predict(x_test,batch_size=32,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 0, 9, ..., 3, 9, 2])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = np.argmax(y_test,axis=1)\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_file = \"submission_v1.csv\"\n",
    "with open(output_file, 'w') as f :\n",
    "    f.write('ImageId,Label\\n')\n",
    "    for i in range(len(y_pred)) :\n",
    "        f.write(\"\".join([str(i+1),',',str(y_pred[i]),'\\n']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
